import json
import pymongo
import requests
client = pymongo.MongoClient("mongodb://localhost:27017")
db = client.Advanced
collection = db.qq
url = [
    "24hours",
    "antip",
    "bj",
    "sh",
    "tj",
    "cq",
    "qinghai",
    "ln",
    "guizhou",
    "guangxi",
    "hainan",
    "gansu",
    "sd",
    "jangxi",
    "ningxia",
    "hn",
    "hebei",
    "heilongjiang",
    "fj",
    "neimenggu",
    "anhui",
    "xian",
    "shanxi",
    "jiangsu",
    "henan",
    "zj",
    "yn",
    "ent",
    "milite",
    "world",
    "cul",
    "finance_stock",
    "house",
    "comic",
    "emotion",
    "digi",
    "astro",
    "health",
    "baby",
    "history",
    "zfw",
    "football",
    "society",
    "cul_ru",
    "edu",
    "finance_licai",
    "topic",
    "sports",
    "life",
    "kepu",
]
all = []
def add_url(name):
    url = 'https://i.news.qq.com/trpc.qqnews_web.kv_srv.kv_srv_http_proxy/list?sub_srv_id=%s&srv_id=pc&offset=%d&limit=20&strategy=1&ext={"pool":["top"],"is_filter":7,"check_type":true}'
    for i in range(1, 11):
        all.append(url % (name, i*20))
for i in url:
    add_url(i)
for i in all:
    res = requests.get(i)
    try:
        jss = res.json()
    except json.decoder.JSONDecodeError as e:
        print(e)
    else:
        try:
            jss = jss['data']['list']
        except TypeError:
            break
        for js in jss:
            try:
                print(js['title'], js['url'], js['update_time'])
            except UnicodeEncodeError:
                pass
            else:
                data = {
                    "title" : js['title'],
                    "url" : js['url'],
                    "update-time" : js['update_time'],
                }
                collection.insert_one(data)
print("Done!")
